/*
 *    Copyright (c) 2014 Xiang Xu <xuxiang@mail.bnu.edu.cn>
 *
 *    Permission is hereby granted, free of charge, to any person
 *    obtaining a copy of this software and associated documentation
 *    files (the "Software"), to deal in the Software without
 *    restriction, including without limitation the rights to use,
 *    copy, modify, merge, publish, distribute, sublicense, and/or sell
 *    copies of the Software, and to permit persons to whom the
 *    Software is furnished to do so, subject to the following
 *    conditions:
 *
 *    The above copyright notice and this permission notice shall be
 *    included in all copies or substantial portions of the Software.
 *
 *    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 *    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 *    OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 *    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 *    HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 *    WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 *    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 *    OTHER DEALINGS IN THE SOFTWARE.
 */

#include "facade_model.h"
#include "scale_model.h"
#include <fstream>
#include <ctime>
#include <boost/progress.hpp>

//#define PRINT_INFO


#define XY2INDEX(w, h, width) ((h) * (width) + (w))

FacadeModel::FacadeModel(int width, int height, boost::shared_ptr<FacadeGrammar> facade_grammar) :
    FacadeFeature(width, height, facade_grammar)
{
    // InitModel();
}

FacadeModel::FacadeModel(boost::shared_ptr<FacadeGrammar> facade_grammar) : FacadeFeature(facade_grammar)
{
}

void FacadeModel::InitModel(int grouping)
{
    state_grouping_ = grouping;

    // Initial state, there are two kinds of states:
    //	state_ = 0 ; 	// @state_ is for learning algorithm, since learning algorithm only need a number

    // @current_state_ contains very detailed state information
    current_state_ = boost::shared_ptr<ShapeSymbol> (new ShapeSymbol(facade_grammar_));
    current_state_->set_position_x(0);
    current_state_->set_position_y(0);
    current_state_->set_width(width_);
    current_state_->set_height(height_);

    current_state_->set_symbol_name(facade_grammar_->get_atom_symbol());

    // Init state and actions discrete
    if(state_grouping_)
        states_number_ = InitGroupingStateMap();
    else
        states_number_  = InitStateMap();

    actions_number_ = InitActionMap();
    //actions_number_ = InitActionMapFromFile();

    InitSymbolSuitableActions();

    RLAlgorithm::Init(states_number_, actions_number_);

    epsilon_ = 0.9;
    gamma_ = 1.0;
    learning_rate_ = 0.1;
    episodes_ = 50000;
}

FacadeModel::~FacadeModel()
{
}

int FacadeModel::InitActionMapFromFile()
{
    int min_size = 1;

    std::ifstream fin(action_parameter_file_.c_str());
    {
        // vertical case
        std::string dummy;
        int size;
        fin >> dummy >> size;
        action_parameters_vertical_.resize(size);

        for(int i = 0; i < size; ++i)
        {
            int action_size;
            fin >> dummy >> action_size;

            for(int j = 0; j < action_size; ++j)
            {
                int parameter;
                fin >> parameter;

                if(parameter < min_size)
                    continue;

                bool already_has = false;

                for(int k = 0; k < action_parameters_vertical_[i].size(); ++k)
                {
                    if(action_parameters_vertical_[i][k] == parameter)
                        already_has = true;
                }

                if(!already_has)
                    action_parameters_vertical_[i].push_back(parameter);
            }
        }
    }

    {
        // horizontal case
        std::string dummy;
        int size;
        fin >> dummy >> size;
        action_parameters_horizontal_.resize(size);

        for(int i = 0; i < size; ++i)
        {
            int action_size;
            fin >> dummy >> action_size;

            for(int j = 0; j < action_size; ++j)
            {
                int parameter;
                fin >> parameter;

                if(parameter < min_size)
                    continue;

                bool already_has = false;

                for(int k = 0; k < action_parameters_horizontal_[i].size(); ++k)
                    if(action_parameters_horizontal_[i][k] == parameter)
                        already_has = true;

                if(!already_has)
                    action_parameters_horizontal_[i].push_back(parameter);
            }
        }
    }

    int count = 0;

    for(int i = 0; i < facade_grammar_->get_grammar_number(); ++i)
    {
        // For each grammar, look for the action ...
        ShapeGrammar sg = facade_grammar_->get_grammar(i);
        std::vector<std::string> terminals = facade_grammar_->get_terminal_symbols();

        std::vector<int> actions;

        if(sg.symbol_parameter_ != "none")
        {
            int index = facade_grammar_->get_terminal_symbol_index(sg.symbol_parameter_);

            if(sg.apply_direction_ == 0)
                actions = action_parameters_vertical_[index];
            else
                actions = action_parameters_horizontal_[index];
        }

        if(sg.value_parameter_.size() != 0)
        {
            for(int j = 0; j < sg.value_parameter_.size(); ++j)
            {
                bool already_has = false;

                for(int k = 0; k < actions.size(); ++k)
                {
                    if(actions[k] == sg.value_parameter_[j])
                        already_has = true;
                }

                if(!already_has)
                    actions.push_back(sg.value_parameter_[j]);
            }
        }

        if(sg.parameter_range_.size() != 0)
        {
            for(int j = sg.parameter_range_[0] + 1; j <= sg.parameter_range_[1]; j += 2)
            {
                bool already_has = false;

                for(int k = 0; k < actions.size(); ++k)
                {
                    if(actions[k] == j)
                        already_has = true;
                }

                if(!already_has)
                    actions.push_back(j);
            }
        }

        //for (int j = 0; j < terminals.size(); ++j) {
        //	if (sg.parameter_ == terminals[j]) {
        //		if (sg.apply_direction_ == 0)
        //			actions = action_parameters_vertical_[j];
        //		else
        //			actions = action_parameters_horizontal_[j];
        //	}

        for(int k = 0; k < actions.size(); ++k)
        {
            SplitAction sa(sg.name_, actions[k], i);
            action_to_index_.insert(std::make_pair(sa, count));
            index_to_action_.insert(std::make_pair(count, sa));
            count++;
        }

        //	}

        // for all the terminal symbols
        //for (int j = 0; j < terminals.size(); ++j) {
        //	if (sg.right_hand_symbol1_ == terminals[j]) {
        //		if (sg.apply_direction_ == 0) { // vertical grammar
        //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!CHECK THIS!!!!!!!!!!!!!
        //			actions = action_parameters_vertical_[j];
        //		} else {
        //			actions = action_parameters_horizontal_[j];
        //		}

        //		for (int k = 0; k < actions.size(); ++k) {
        //			SplitAction sa (sg.grammar_name_, actions[k], i);
        //			action_to_index_.insert(std::make_pair(sa, count));
        //			index_to_action_.insert(std::make_pair(count, sa));
        //			count++;
        //		}
        //	}
        //}
        std::cout << "Init action for grammar " << i << std::endl;
        std::cout << sg.name_ << "\t";

        for(int j = 0; j < actions.size(); ++j)
        {
            std::cout << actions[j] << "\t";
        }

        std::cout << std::endl << std::endl;
    }

    return count;
}

int FacadeModel::InitActionMap()
{
    std::cout << "Initialize action ...\n";

    int count = 0;

    for(int i = 0; i < facade_grammar_->get_grammar_number(); ++i)
    {
        ShapeGrammar sg = facade_grammar_->get_grammar(i);
        std::vector<int> actions;

        if(sg.symbol_parameter_ != "none")
        {
            int min_size = 1;

            std::ifstream fin(action_parameter_file_.c_str());
            {
                // vertical case
                std::string dummy;
                int size;
                fin >> dummy >> size;
                action_parameters_vertical_.resize(size);

                for(int i = 0; i < size; ++i)
                {
                    int action_size;
                    fin >> dummy >> action_size;

                    for(int j = 0; j < action_size; ++j)
                    {
                        int parameter;
                        fin >> parameter;

                        if(parameter < min_size)
                            continue;

                        bool already_has = false;

                        for(int k = 0; k < action_parameters_vertical_[i].size(); ++k)
                        {
                            if(action_parameters_vertical_[i][k] == parameter)
                                already_has = true;
                        }

                        if(!already_has)
                            action_parameters_vertical_[i].push_back(parameter);
                    }
                }
            }

            {
                // horizontal case
                std::string dummy;
                int size;
                fin >> dummy >> size;
                action_parameters_horizontal_.resize(size);

                for(int i = 0; i < size; ++i)
                {
                    int action_size;
                    fin >> dummy >> action_size;

                    for(int j = 0; j < action_size; ++j)
                    {
                        int parameter;
                        fin >> parameter;

                        if(parameter < min_size)
                            continue;

                        bool already_has = false;

                        for(int k = 0; k < action_parameters_horizontal_[i].size(); ++k)
                            if(action_parameters_horizontal_[i][k] == parameter)
                                already_has = true;

                        if(!already_has)
                            action_parameters_horizontal_[i].push_back(parameter);
                    }
                }
            }


            int index = facade_grammar_->get_terminal_symbol_index(sg.symbol_parameter_);

            if(sg.apply_direction_ == 0)
                actions = action_parameters_vertical_[index];
            else
                actions = action_parameters_horizontal_[index];
        }

        // value parameters
        if(sg.value_parameter_.size() != 0)
        {
            for(int j = 0; j < sg.value_parameter_.size(); ++j)
            {
                bool already_has = false;

                for(int k = 0; k < actions.size(); ++k)
                {
                    if(actions[k] == sg.value_parameter_[j])
                        already_has = true;
                }

                if(!already_has)
                    actions.push_back(sg.value_parameter_[j]);
            }
        }

        // parameter range
        if(sg.parameter_range_.size() != 0)
        {
            for(int j = sg.parameter_range_[0] + 1; j <= sg.parameter_range_[1]; j += 2)
            {
                bool already_has = false;

                for(int k = 0; k < actions.size(); ++k)
                {
                    if(actions[k] == j)
                        already_has = true;
                }

                if(!already_has)
                    actions.push_back(j);
            }
        }

        // Add the actions
        for(int k = 0; k < actions.size(); ++k)
        {
            SplitAction sa(sg.name_, actions[k], i);
            action_to_index_.insert(std::make_pair(sa, count));
            index_to_action_.insert(std::make_pair(count, sa));
            count++;
        }

        std::cout << "Init action for grammar " << i << std::endl;
        std::cout << sg.name_ << "\t";

        for(int j = 0; j < actions.size(); ++j)
        {
            std::cout << actions[j] << "\t";
        }

        std::cout << std::endl << std::endl;

    }

    assert(action_to_index_.size() == count && index_to_action_.size() == count);

    std::cout << "Done. " << count << " actions\n";

#ifdef PRINT_INFO
    std::ofstream ofs("action_index.txt");

    for(int i = 0; i < count; ++i)
    {
        SplitAction sa = index_to_action_.at(i);
        ofs << i << ": " << sa.get_grammar_name() << "\t" << sa.get_parameter() << "\n";
    }

    ofs.close();
#endif

    return count;
}

int FacadeModel::InitGroupingStateMap()
{
    int dim = width_ > height_ ? width_ : height_;
    int count = 0;

    // atom state
    Facadestate fs(facade_grammar_->get_atom_symbol(), 0);
    state_to_index_.insert(std::make_pair(fs, count));
    index_to_state_.insert(std::make_pair(count, fs));

    count++;

    for(int pos = 0; pos < dim; ++pos)	 	// position along the split direction
    {
        // unterminal_symbol_
        for(int shape_id = 0; shape_id < facade_grammar_->get_unterminal_symbol_size(); ++shape_id)
        {
            Facadestate fs(facade_grammar_->get_unterminal_symbol(shape_id), pos);
            state_to_index_.insert(std::make_pair(fs, count));
            index_to_state_.insert(std::make_pair(count, fs));

            count++;
        }

        //// terminal_symbol_
        //for (int shape_id = 0; shape_id < facade_grammar_->get_terminal_symbol_size(); ++shape_id) {
        //	Facadestate fs (facade_grammar_->get_terminal_symbol(shape_id), pos);
        //	state_to_index_.insert(std::make_pair(fs, count));
        //	index_to_state_.insert(std::make_pair(count, fs));

        //	count++;
        //}
    }

    return count;
}

int FacadeModel::InitStateMap()
{
    std::cout << "Initialize state ...\n";
    boost::progress_display display(height_ * width_ * (1 + facade_grammar_->get_unterminal_symbol_size()));
    //	+ facade_grammar_->get_terminal_symbol_size()));

    // record #states
    int count = 0;

    for(int h = 0; h < height_; ++h)
    {
        for(int w = 0; w < width_; ++w)
        {
            //int pos = h * width_ + w;
            int pos = XY2INDEX(w, h, width_);

            // atom state
            Facadestate fs(facade_grammar_->get_atom_symbol(), pos);
            state_to_index_.insert(std::make_pair(fs, count));
            index_to_state_.insert(std::make_pair(count, fs));
            ++count;
            ++display;

            // unterminal_symbol_
            for(int shape_id = 0; shape_id < facade_grammar_->get_unterminal_symbol_size(); ++shape_id)
            {
                Facadestate fs(facade_grammar_->get_unterminal_symbol(shape_id), pos);
                state_to_index_.insert(std::make_pair(fs, count));
                index_to_state_.insert(std::make_pair(count, fs));

                count++;
                ++display;
            }

#if 0 // NO Need!

            // terminal_symbol_
            for(int shape_id = 0; shape_id < facade_grammar_->get_terminal_symbol_size(); ++shape_id)
            {
                Facadestate fs(facade_grammar_->get_terminal_symbol(shape_id), pos);
                state_to_index_.insert(std::make_pair(fs, count));
                index_to_state_.insert(std::make_pair(count, fs));

                count++;
                ++display;
            }

#endif
        }
    }

    assert(state_to_index_.size() == count && index_to_state_.size() == count);
    std::cout << "Done. " << count << " states.\n";

#ifdef PRINT_INFO
    std::ofstream ofs("state_index1.txt");

    for(int i = 0; i < count; ++i)
    {
        Facadestate sa = index_to_state_.at(i);
        int height = (int)(sa.get_parameter()) / width_;
        int width = (int)(sa.get_parameter()) % width_;
        ofs << i << ": " << sa.get_grammar_name() << "\t" << width << "\t" << height << "\n";
    }

    ofs.close();

    ofs.open("state_index2.txt");

    for(int h = 0; h < height_; ++h)
    {
        for(int w = 0; w < width_; ++w)
        {
            int pos = h * width_ + w;
            Facadestate state(facade_grammar_->get_atom_symbol(), pos);
            int index = state_to_index_.at(state);
            ofs << "[" << facade_grammar_->get_atom_symbol() << ", " << w << ", " << h << "] = " << index <<
                std::endl;
        }
    }

    for(int h = 0; h < height_; ++h)
    {
        for(int w = 0; w < width_; ++w)
        {
            int pos = h * width_ + w;

            for(int i = 0; i < facade_grammar_->get_unterminal_symbol_size(); ++i)
            {
                Facadestate state(facade_grammar_->get_unterminal_symbol(i), pos);
                int index = state_to_index_.at(state);
                ofs << "[" << facade_grammar_->get_unterminal_symbol(i) << ", " << w << ", " << h << "] = " <<
                    index << std::endl;
            }
        }
    }

#if 0

    for(int h = 0; h < height_; ++h)
    {
        for(int w = 0; w < width_; ++w)
        {
            int pos = h * width_ + w;

            for(int i = 0; i < facade_grammar_->get_terminal_symbol_size(); ++i)
            {
                Facadestate state(facade_grammar_->get_terminal_symbol(i), pos);
                int index = state_to_index_.at(state);
                ofs << "[" << facade_grammar_->get_terminal_symbol(i) << ", " << w << ", " << h << "] = " << index
                    << std::endl;
            }
        }
    }

#endif
    ofs.close();

#endif

    return count;
}

void FacadeModel::InitSymbolSuitableActions()
{
    std::cout << "Initialize suitable actions for state ...\n";

    // atom symbol
    std::vector<int> actions_for_atoms;
    std::vector<std::vector<int> > actions_for_unterminal;
    actions_for_unterminal.resize(facade_grammar_->get_unterminal_symbol_size());

    for(int i = 0; i < actions_number_; ++i)
    {
        SplitAction action = index_to_action_.at(i);
        ShapeGrammar grammar = facade_grammar_->get_grammar(action.get_shape_grammar_index());

        if(grammar.left_hand_symbol_ == facade_grammar_->get_atom_symbol())
        {
            actions_for_atoms.push_back(i);

            goto START_NEW_LOOP;
        }


        for(int j = 0; j < facade_grammar_->get_unterminal_symbol_size(); ++j)
        {
            if(grammar.left_hand_symbol_ == facade_grammar_->get_unterminal_symbol(j))
            {
                actions_for_unterminal[j].push_back(i);

                goto START_NEW_LOOP;
            }
        }

    START_NEW_LOOP:
        ;
    }

    action_index_for_symbol_.insert(std::make_pair(facade_grammar_->get_atom_symbol(), actions_for_atoms));

    for(int i = 0; i < facade_grammar_->get_unterminal_symbol_size(); ++i)
        action_index_for_symbol_.insert(std::make_pair(facade_grammar_->get_unterminal_symbol(i),
                                        actions_for_unterminal[i]));

    std::cout << "Done.\n";

#ifdef PRINT_INFO
    // print something
	 std::cout << "there are " << action_index_for_symbol_.size() << " items in action_index_for_symbol_.\n";
	 std::vector<int> actions = action_index_for_symbol_.at(facade_grammar_->get_atom_symbol());
	 std::cout << "#actions for " << facade_grammar_->get_atom_symbol() << " " << actions.size() << std::endl;
	 for (int i = 0; i < facade_grammar_->get_unterminal_symbol_size(); ++i) {
		actions.clear();
		actions = action_index_for_symbol_.at(facade_grammar_->get_unterminal_symbol(i));
		std::cout << "#actions for " << facade_grammar_->get_unterminal_symbol(i) << " " << actions.size() << std::endl;
	 }
#endif
}

void FacadeModel::RandomParsing()
{
    Reset();

    epsilon_ = 0.0;     // always random
    learning_ = false; // not do learning
    record_ = true;     // record result

    RandomTask(current_state_, 0);
}

double FacadeModel::RunParsingAlgorithm(int episodes, int greedy_episodes, double learning_rate)
{
    episodes_ = episodes;
    unsigned int rand_seed = static_cast<unsigned int>(std::time(0));
    std::srand(rand_seed);
    learning_ = true;
    record_ = false;
    int total_pixel_number;
    learning_rate_ = learning_rate;
    epsilon_ = 0.1;

    // Learning @episodes_ times ...
    std::cout << "Reinforcement Learning starts here...\n";

    for(int i = 0; i < episodes_; ++i)
    {
        // Calculate the parameters
        double seed = (double) i / (double) episodes_;
        epsilon_ = ScaleModel::ExpTrans(seed, 0.001, 0.999);
        // gamma_ = 1 - epsilon_;
        // learning_rate_ = 1 - epsilon_;

        double score = 0.0;
        // Reset the environment(recover the parsing original)
        Reset();
        total_pixel_number = current_state_->width() * current_state_->height();

        // Parsing from root state
        score = RunTask(current_state_, 0);

        score /= total_pixel_number;

        if(i % 1000 == 0)
        {
            std::cout << "\nNow In Episode " << i << ". score = " << score <<
                      " epsilon: " << epsilon_ << " learning_rate_: " << learning_rate_ /*<< " Seed: " << rand_seed*/ <<
                      std::endl;
        }

        if(i % 10000 == 0)
        {
            //double back_learning_rate = learning_rate_;
            double back_epsilon = epsilon_;

            //learning_rate_ = 0.01;
            epsilon_ = 1.0;
            learning_ = false;

            Reset();
            double score = RunTask(current_state_, 0);
            std::cout << "\n\nGreedy Score: " << score / total_pixel_number << std::endl;

            //learning_rate_ = back_learning_rate;
            epsilon_ = back_epsilon;
            learning_ = true;
        }
    }

#if 1
    // Run some episodes with big greedy ...
    epsilon_ = 0.9;
    gamma_ = 0.1;

    for(int i = 0; i < greedy_episodes; ++i)
    {
        double score = 0.0;
        // Reset the environment(recover the parsing original)
        Reset();
        total_pixel_number = current_state_->width() * current_state_->height();

        // Parsing from root state
        score = RunTask(current_state_, 0);

        if(i % 10000 == 0)
        {
            std::cout << "\nEpisode " << i << " Score: " << score / total_pixel_number <<
                      " epsilon: " << epsilon_ << " gamma: " << gamma_ /*<< " Seed: " << rand_seed*/ << std::endl;

        }
    }

#endif
#if 0
    epsilon_ = 0.99;
//	gamma_ = 1.0;
//	learning_rate_ = 0.01;

    const int episodes_per_step = 100;
    int episodes_count = 1;
    current_average_ = 0;
    const double threshold = 0.0001;

    do
    {
        last_average_ = current_average_;
        current_average_ = 0.0;

        for(int i = 0; i < episodes_per_step; ++i)
        {
            double score = 0.0;

            // Reset the environment(recover the parsing original)
            Reset();
            total_pixel_number = current_state_->width() * current_state_->height();

            // Parsing from root state
            score = RunTask(current_state_, 0);

            score /= total_pixel_number;
            current_average_ += score;
            ++episodes_count;
        }

        current_average_ /= episodes_per_step;
        std::cout << "\nNow In Episode " << episodes_count << ". Last "
                  << episodes_per_step << " episodes average score = " << current_average_;

    }
    while((std::abs(last_average_ - current_average_) >= threshold) && (episodes_count < greedy_episodes));

#endif
// Get the greedy solution
    Reset();
    epsilon_ = 1.0;
    learning_ = false;
    record_ = true;
    double score = RunTask(current_state_, 0);
    std::cout << "\nGreedy Score: " << score / total_pixel_number << std::endl;

// Save the learning result.
    SaveQTable(qtable_path.c_str());

    return (score / total_pixel_number);

}

void FacadeModel::RandomTask(boost::shared_ptr<ShapeSymbol>& state, int state_index)
{
    int action, next_state;

    while(!state->is_terminal())
    {
        bool end_of_task = false;

        if(state_grouping_)
        {
            if(state_action_map_.find(state_index) != state_action_map_.end())
            {
                action = state_action_map_.at(state_index);
            }
            else
            {
                std::vector<int> suitable_actions = action_index_for_symbol_.at(state->symbol_name());
                action = suitable_actions[rand() % suitable_actions.size()];
                state_action_map_.insert(std::make_pair(state_index, action));
            }
        }
        else
        {
            std::vector<int> suitable_actions = action_index_for_symbol_.at(state->symbol_name());
            action = suitable_actions[rand() % suitable_actions.size()];
            state_action_map_.insert(std::make_pair(state_index, action));
        }

        // Get the very detailed action
        SplitAction split_action = index_to_action_.at(action);

        if(record_)
        {
            std::ostringstream oss;
            oss << split_action.get_grammar_name() << " [ " <<
                split_action.get_parameter() << " ]";
            grammar_result_.push_back(oss.str());
        }

        ShapeGrammar shape_grammar = facade_grammar_->get_grammar(split_action.get_shape_grammar_index());
        boost::shared_ptr<ShapeSymbol> right_symbol1(new ShapeSymbol(facade_grammar_));
        boost::shared_ptr<ShapeSymbol> right_symbol2(new ShapeSymbol(facade_grammar_));
        bool split_result = state->DoSplit(shape_grammar, split_action.get_parameter(), right_symbol1, right_symbol2);

        if(split_result)
        {

            if(!right_symbol1->is_terminal() && right_symbol2->is_terminal())
            {
                // exchange @right_symbol1 and @right_symbol2
                ShapeSymbol tmp(right_symbol1->position_x(), right_symbol1->position_y(),
                                right_symbol1->width(), right_symbol1->height(), right_symbol1->symbol_name(),
                                facade_grammar_);

                right_symbol1->SetScope(right_symbol2->position_x(), right_symbol2->position_y(),
                                        right_symbol2->width(), right_symbol2->height());
                right_symbol1->set_symbol_name(right_symbol2->symbol_name());

                right_symbol2->SetScope(tmp.position_x(), tmp.position_y(),
                                        tmp.width(), tmp.height());
                right_symbol2->set_symbol_name(tmp.symbol_name());
            }

            if(right_symbol1->is_terminal())
            {
                if(record_)
                {
                    split_result_.push_back(right_symbol1);
                }
            }
            else
            {
                // Get the sub-state index
                int sub_state;

                if(!state_grouping_)
                {
                    Facadestate facade_state(right_symbol1->symbol_name(),
                                             XY2INDEX(right_symbol1->position_x(), right_symbol1->position_y(),
                                                      width_));
                    sub_state = state_to_index_.at(facade_state);
                }
                else
                {
                    int dir = facade_grammar_->get_symbol_apply_direction(right_symbol1->symbol_name());

                    if(dir == 0)    // vertical
                    {
                        Facadestate facade_state(right_symbol1->symbol_name(), right_symbol1->position_y());
                        sub_state = state_to_index_.at(facade_state);
                    }
                    else
                    {
                        Facadestate facade_state(right_symbol1->symbol_name(), right_symbol1->position_x());
                        sub_state = state_to_index_.at(facade_state);
                    }
                }

                RandomTask(right_symbol1, sub_state);
            }

            // Get the state after executing the action
            if(right_symbol2->is_terminal())
            {
                end_of_task = true; // If two child are all terminal, it arrives at parsing terminal.
            }
            else
            {
                if(!state_grouping_)
                {
                    Facadestate facade_state(right_symbol2->symbol_name(),
                                             XY2INDEX(right_symbol2->position_x(), right_symbol2->position_y(),
                                                      width_));
                    next_state = state_to_index_.at(facade_state);
                }
                else
                {
                    int dir = facade_grammar_->get_symbol_apply_direction(right_symbol2->symbol_name());

                    if(dir == 0)    // vertical
                    {
                        Facadestate facade_state(right_symbol2->symbol_name(), right_symbol2->position_y());
                        next_state = state_to_index_.at(facade_state);
                    }
                    else
                    {
                        Facadestate facade_state(right_symbol2->symbol_name(), right_symbol2->position_x());
                        next_state = state_to_index_.at(facade_state);
                    }
                }
            }

            state = right_symbol2;
        }
        else
        {
            if(record_)
            {
                split_result_.push_back(right_symbol1);
            }

            end_of_task = true;
            state = right_symbol1;
        }

        state_index = next_state;
    }
}

double FacadeModel::RunTask(boost::shared_ptr<ShapeSymbol>& state, int state_index)
{
    double episode_score = 0.0;
    int action, next_state;

    while(!state->is_terminal())
    {
        double score = 0.0;
        bool end_of_task = false;

        // Explore an action for current state
        std::vector<int> suitable_actions = action_index_for_symbol_.at(state->symbol_name());
		  if (suitable_actions.size() == 0) 
			  int a = 0;
        Explore(state_index, action, epsilon_, suitable_actions);

        // Get the very detailed action
        SplitAction split_action = index_to_action_.at(action);

        if(record_)
        {
            std::ostringstream oss;
            oss << split_action.get_grammar_name() << " [ " <<
                split_action.get_parameter() << " ]";
            grammar_result_.push_back(oss.str());
        }

        boost::shared_ptr<ShapeSymbol> right_symbol1(new ShapeSymbol(facade_grammar_));
        boost::shared_ptr<ShapeSymbol> right_symbol2(new ShapeSymbol(facade_grammar_));
		  ShapeGrammar shape_grammar = facade_grammar_->get_grammar(split_action.get_shape_grammar_index());
        bool split_result = state->DoSplit(shape_grammar, split_action.get_parameter(), right_symbol1, right_symbol2);
        int pixel_number = right_symbol1->height() * right_symbol1->width();

        if(split_result)
        {
            if(!right_symbol1->is_terminal() && right_symbol2->is_terminal())
            {
                // exchange @right_symbol1 and @right_symbol2
                ShapeSymbol tmp(right_symbol1->position_x(), right_symbol1->position_y(),
                                right_symbol1->width(), right_symbol1->height(), right_symbol1->symbol_name(),
                                facade_grammar_);

                right_symbol1->SetScope(right_symbol2->position_x(), right_symbol2->position_y(),
                                        right_symbol2->width(), right_symbol2->height());
                right_symbol1->set_symbol_name(right_symbol2->symbol_name());

                right_symbol2->SetScope(tmp.position_x(), tmp.position_y(),
                                        tmp.width(), tmp.height());
                right_symbol2->set_symbol_name(tmp.symbol_name());
            }

            if(right_symbol1->is_terminal())
            {
                // calculate the score
                score += CalculateScore(right_symbol1);

                if(record_)
                {
                    split_result_.push_back(right_symbol1);
                }
            }
            else
            {
                // Get the sub-state index
                int sub_state;

                if(!state_grouping_)
                {
                    Facadestate facade_state(right_symbol1->symbol_name(),
                                             XY2INDEX(right_symbol1->position_x(), right_symbol1->position_y(),
                                                      width_));
                    sub_state = state_to_index_.at(facade_state);
                }
                else
                {
                    int dir = facade_grammar_->get_symbol_apply_direction(right_symbol1->symbol_name());

                    if(dir == 0)    // vertical
                    {
                        Facadestate facade_state(right_symbol1->symbol_name(), right_symbol1->position_y());
                        sub_state = state_to_index_.at(facade_state);
                    }
                    else
                    {
                        Facadestate facade_state(right_symbol1->symbol_name(), right_symbol1->position_x());
                        sub_state = state_to_index_.at(facade_state);
                    }
                }

                score += RunTask(right_symbol1, sub_state);
            }

            // Get the state after executing the action
            if(right_symbol2->is_terminal())
            {
                score += CalculateScore(right_symbol2);
                pixel_number += (right_symbol2->width() * right_symbol2->height());

                end_of_task = true; // If two child are all terminal, it arrives at parsing terminal.
            }
            else
            {
                if(!state_grouping_)
                {
                    Facadestate facade_state(right_symbol2->symbol_name(),
                                             XY2INDEX(right_symbol2->position_x(), right_symbol2->position_y(),
                                                      width_));
                    next_state = state_to_index_.at(facade_state);
                }
                else
                {
                    int dir = facade_grammar_->get_symbol_apply_direction(right_symbol2->symbol_name());

                    if(dir == 0)    // vertical
                    {
                        Facadestate facade_state(right_symbol2->symbol_name(), right_symbol2->position_y());
                        next_state = state_to_index_.at(facade_state);
                    }
                    else
                    {
                        Facadestate facade_state(right_symbol2->symbol_name(), right_symbol2->position_x());
                        next_state = state_to_index_.at(facade_state);
                    }
                }
            }

            state = right_symbol2;
        }
        else
        {
            score += CalculateScore(right_symbol1);

            if(record_)
            {
                split_result_.push_back(right_symbol1);
            }

            end_of_task = true;
            state = right_symbol1;
        }

        episode_score += score; // to return
        score /= pixel_number;  // for learning

        if(learning_)
        {
            // update state-action value pair
            // When @end_of_task == true, @next_state_ is not used.
            Update(state_index, action, score, next_state, end_of_task, learning_rate_, gamma_);
        }

        state_index = next_state;
    }

    return episode_score;
}


double FacadeModel::CalculateScore(boost::shared_ptr<ShapeSymbol> shape_symbol)
{
    double score = 0.0;
    int nterminals = facade_grammar_->get_terminal_symbol_size();
    int symbol_index = facade_grammar_->get_terminal_symbol_index(shape_symbol->symbol_name());

    int height_begin = shape_symbol->position_y();
    int height_end = height_begin + shape_symbol->height();
    int width_begin = shape_symbol->position_x();
    int width_end = width_begin + shape_symbol->width();

    for(int h = height_begin; h < height_end; ++h)
    {
        for(int w = width_begin; w < width_end; ++w)
        {
#if 0
            // Version 1
            score += grd_[FacadeFeature::FEATURE_NUMBER + symbol_index + 1][h][w];

            if(facade_grammar_->has_empty_symbol())
                grd_[FacadeFeature::FEATURE_NUMBER + symbol_index][h][w];

#endif

            //  Version 2
            if(!facade_grammar_->has_empty_symbol())
            {
                if(grd_[FEATURE_NUMBER][h][w] == symbol_index)
                {
                    score += weights_[symbol_index];
                    //++score;
                }
            }
            else
            {
                if(grd_[FEATURE_NUMBER][h][w] == symbol_index ||
                        grd_[FEATURE_NUMBER][h][w] == nterminals - 1)
                {
                    score += weights_[symbol_index];
                    // ++score;
                }
            }
        }
    }

    return score;
}

void FacadeModel::Reset()
{
    // Initial state, there are two kinds of states:
    //	state_ = 0 ; 	// @state is for learning algorithm, since learning algorithm only need a number
    //	next_state_ = 0;

    // @current_state_ contains very detailed state information
    current_state_->set_position_x(0);
    current_state_->set_position_y(0);
    current_state_->set_width(width_);
    current_state_->set_height(height_);

    current_state_->set_symbol_name(facade_grammar_->get_atom_symbol());

    //  terminal_symbol_.clear();
    split_result_.clear();
    result_is_exception_.clear();
}

void FacadeModel::ConfigureOneDimensionFacade(std::string path)
{
    for(int x  = 0; x < 4; ++x)
    {
        for(int y = 0; y < 5; ++y)
        {
            SetProperty(x, y, 2, 0.0);  // WINDOW
        }
    }

    for(int x  = 4; x < 7; ++x)
    {
        for(int y = 0; y < 5; ++y)
        {
            SetProperty(x, y, 2, 1.0);  // wall
        }
    }

    for(int x  = 7; x < 10; ++x)
    {
        for(int y = 0; y < 5; ++y)
        {
            SetProperty(x, y, 2, 0.0);  // window
        }
    }

    RenderGrds(path);
}

void FacadeModel::ConfigureComplexOneDimensioFacade(std::string path)
{
    int pos = 0;

    while(pos != 200)
    {
        SetSymbol(pos, pos + 10, 0, 100, 1);  // wall
        pos += 10;
        SetSymbol(pos, pos + 10, 0, 100, 0);  // window
        pos += 10;
    }

    SetSymbol(200, 210, 0, 100, 1);  // wall

    RenderGrds(path);
}

void FacadeModel::ConfigureTwoDimensionFacade(std::string path)
{
    // Set Wall
    SetSymbol(0, 50, 50, 100, 1);
    SetSymbol(0, 50, 150, 200, 1);

    SetSymbol(100, 150, 50, 100, 1);
    SetSymbol(100, 150, 150, 200, 1);

    SetSymbol(200, 250, 50, 100, 1);
    SetSymbol(200, 250, 150, 200, 1);

    SetSymbol(0, 250, 0, 50, 1);
    SetSymbol(0, 250, 100, 150, 1);
    SetSymbol(0, 250, 200, 250, 1);

    // Set Window
    SetSymbol(50, 100, 50, 100, 0);
    SetSymbol(50, 100, 150, 200, 0);
    SetSymbol(150, 200, 50, 100, 0);
    SetSymbol(150, 200, 150, 200, 0);

    RenderGrds(path);
}

void FacadeModel::RenderResult(std::string& filename)
{
    cv::Mat facade_image = cv::Mat::zeros(height_, width_, CV_8UC3);

    for(size_t i = 0; i < split_result_.size(); ++i)
    {
        cv::Point p1(split_result_[i]->position_x(), height_ - split_result_[i]->position_y() - 1);
        cv::Point p2(p1.x + split_result_[i]->width(), height_ - 1 - (split_result_[i]->position_y() +
                     split_result_[i]->height()));
        std::vector<double> symbol_color = get_symbol_color(split_result_[i]->symbol_name());
        cv::Scalar color(symbol_color[0] * 255, symbol_color[1] * 255, symbol_color[2] * 255);

        cv::rectangle(facade_image, p1, p2, color, -1);
    }

    cv::imwrite(filename, facade_image);
}

void FacadeModel::PrintQTable()
{

}

void FacadeModel::SaveEasyReadQTable()
{
}


void FacadeModel::SaveLearningResult(std::string path, double score)
{
    std::ofstream fout(path.c_str());

    fout << score << std::endl;       // score
    fout << split_result_.size() << std::endl; // #terminal symbols
    fout << extents_min_.x << " " << extents_min_.y << " " << resolution_ << " " << width_ << " " <<
         height_ << std::endl; // extends

    // symbols
    for(size_t i = 0; i < split_result_.size(); ++i)
    {
        fout << split_result_[i]->symbol_name() << " " <<
             split_result_[i]->position_x() << " " << split_result_[i]->position_y() <<
             " " << split_result_[i]->width() << " " << split_result_[i]->height() << "\n";
    }


    // grammars
    for(size_t i = 0; i < grammar_result_.size(); ++i)
    {
        fout << grammar_result_[i] << std::endl;
    }

    fout.close();
}
